Xinzhu Wang(xw2581), Ying Jin(yj2453), Jincheng Xu(jx2365), Xudong Guo(xg2305)
import os
import numpy as np
import scipy.io
import imageio
import tensorflow as tf
import matplotlib.pyplot as plt
import random
from google.colab import files
from PIL import Image, ImageOps
import cv2
#import necessary packages
print("TF version: ",tf.__version__)
from google.colab import drive
drive.mount('/content/drive')
def resize_image(image_path, width, height, save = True):
image = Image.open(image_path)
image = ImageOps.fit(image, (width, height), Image.ANTIALIAS)
if save:
image_directory = image_path.split('/')
image_directory[-1] = 'resized_' + image_directory[-1]
output_path = '/'.join(image_directory)
if not os.path.exists(output_path):
image.save(output_path)
image = np.asarray(image, np.float32)
return np.expand_dims(image, 0)
#define resize image function
def noise_image(content_image, width, height, noise_ratio = 0.6):
noise_image = np.random.uniform(-20, 20, (1, height, width, 3)).astype(np.float32)
aggregated = noise_image * noise_ratio + content_image * (1 - noise_ratio)
return aggregated
#define noise image function
def preprocess(content_image_path,style_image_path):
content_image = resize_image(content_image_path, image_width, image_height)
style_image = resize_image(style_image_path, image_width, image_height)
noise_images = noise_image(content_image, image_width, image_height)
style_image -= MEAN_PIXEL
content_image -= MEAN_PIXEL
return style_image, content_image, noise_images
#define preprocess function
def inp(image_widths, image_heights):
with tf.variable_scope("input"):
input_img = tf.get_variable("in_img",
shape = ([1, image_heights, image_widths, 3]),
dtype = tf.float32,
initializer = tf.zeros_initializer())
return input_img
#define input_image function
VGG19 = scipy.io.loadmat('drive/My Drive/5242_project/imagenet-vgg-verydeep-19.mat')
vgg_layer = VGG19['layers']
def weights(layer, expected_layer_name):
# Return the weights and bias from VGG for a given layer
weight = vgg_layer[0][layer][0][0][2][0][0] # (3,3,3,64)
bias = vgg_layer[0][layer][0][0][2][0][1] # (64,1)
layer_name = vgg_layer[0][layer][0][0][0][0] #current layer name
assert layer_name == expected_layer_name, print(expected_layer_name, layer_name)
return weight, bias
def conv_relu(previous_layer, current_layer, layer_name):
w, b = weights(current_layer, layer_name)
W = tf.constant(w)
B = tf.constant(np.reshape(b, b.size)) # 64
conv = tf.nn.conv2d(previous_layer, filters=W, strides=[1,1,1,1], padding='SAME')
return tf.nn.relu(conv + B)
def avg_pool(previous_layer):
return tf.nn.avg_pool(previous_layer, ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME')
layers = (
'conv1_1', 'relu1_1', 'conv1_2', 'relu1_2', 'pool1',
'conv2_1', 'relu2_1', 'conv2_2', 'relu2_2', 'pool2',
'conv3_1', 'relu3_1', 'conv3_2', 'relu3_2', 'conv3_3',
'relu3_3', 'conv3_4', 'relu3_4', 'pool3',
'conv4_1', 'relu4_1', 'conv4_2', 'relu4_2', 'conv4_3',
'relu4_3', 'conv4_4', 'relu4_4', 'pool4',
'conv5_1', 'relu5_1', 'conv5_2', 'relu5_2', 'conv5_3',
'relu5_3', 'conv5_4', 'relu5_4'
)
def vgg_model(inputs):
vgg = {}
layer_dict = {}
for layer_idx,layer_name in enumerate(layers):
layer_dict[layer_name]=layer_idx
vgg['input'] = inputs
vgg['conv1_1']=conv_relu(vgg['input'],layer_dict['conv1_1'],'conv1_1')
vgg['conv1_2']=conv_relu(vgg['conv1_1'],layer_dict['conv1_2'],'conv1_2')
vgg['avgpool1']=avg_pool(vgg['conv1_2'])
vgg['conv2_1']=conv_relu(vgg['avgpool1'],layer_dict['conv2_1'],'conv2_1')
vgg['conv2_2']=conv_relu(vgg['conv2_1'],layer_dict['conv2_2'],'conv2_2')
vgg['avgpool2']=avg_pool(vgg['conv2_2'])
vgg['conv3_1']=conv_relu(vgg['avgpool2'],layer_dict['conv3_1'],'conv3_1')
vgg['conv3_2']=conv_relu(vgg['conv3_1'],layer_dict['conv3_2'],'conv3_2')
vgg['conv3_3']=conv_relu(vgg['conv3_2'],layer_dict['conv3_3'],'conv3_3')
vgg['conv3_4']=conv_relu(vgg['conv3_3'],layer_dict['conv3_4'],'conv3_4')
vgg['avgpool3']=avg_pool(vgg['conv3_4'])
vgg['conv4_1']=conv_relu(vgg['avgpool3'],layer_dict['conv4_1'],'conv4_1')
vgg['conv4_2']=conv_relu(vgg['conv4_1'],layer_dict['conv4_2'],'conv4_2')
vgg['conv4_3']=conv_relu(vgg['conv4_2'],layer_dict['conv4_3'],'conv4_3')
vgg['conv4_4']=conv_relu(vgg['conv4_3'],layer_dict['conv4_4'],'conv4_4')
vgg['avgpool4']=avg_pool(vgg['conv4_4'])
vgg['conv5_1']=conv_relu(vgg['avgpool4'],layer_dict['conv5_1'],'conv5_1')
vgg['conv5_2']=conv_relu(vgg['conv5_1'],layer_dict['conv5_2'],'conv5_2')
vgg['conv5_3']=conv_relu(vgg['conv5_2'],layer_dict['conv5_3'],'conv5_3')
vgg['conv5_4']=conv_relu(vgg['conv5_3'],layer_dict['conv5_4'],'conv5_4')
vgg['avgpool5']=avg_pool(vgg['conv5_4'])
return vgg
#define vgg19 model
content_layer = "conv4_2"
style_layers = ["conv1_1", "conv2_1", "conv3_1", "conv4_1", "conv5_1"]
def _content_loss(P, F):
content_loss = tf.reduce_sum(tf.square(F - P)) / 2
return content_loss
def _gram_matrix(F, N, M):
F = tf.reshape(F, (M, N))
return tf.matmul(tf.transpose(F), F)
def _single_style_loss(a, g):
N = a.shape[3]
M = a.shape[1] * a.shape[2]
A = _gram_matrix(a, N, M)
G = _gram_matrix(g, N, M)
return tf.reduce_sum(tf.square(G - A)) / ((2 * N * M) ** 2)
def _style_loss(A):
n_layers = len(A)
E = [_single_style_loss(A[i], vgg[style_layers[i]]) for i in range(n_layers)]
style_loss = sum(style_layers_weights[i] * E[i] for i in range(n_layers))
return style_loss
def losses():
with tf.variable_scope("losses"):
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(input_img.assign(content_image))
gen_img_content = vgg[content_layer]
content_img_content = sess.run(gen_img_content)
content_loss = _content_loss(content_img_content, gen_img_content)
with tf.Session() as sess:
sess.run(input_img.assign(style_image))
new_style_layers = sess.run([vgg[layer] for layer in style_layers])
style_loss = _style_loss(new_style_layers)
total_loss = content_loss_weight * content_loss + style_loss_weight * style_loss
return content_loss, style_loss, total_loss
# define the loss function
def train(iteration=100, seed = 2019, show = False):
tf.set_random_seed(seed)
learning_rate = 2.0
optimizer = tf.train.AdamOptimizer(learning_rate)
training = optimizer.minimize(total_loss)
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
sess.run(input_img.assign(noise_images))
for epoch in range(iteration):
sess.run(training)
# generate a picture for every 20 epoches
if epoch == 0 or (epoch+1) % 20 == 0:
generated_image, total_losses = sess.run([input_img, total_loss])
generated_image = generated_image + MEAN_PIXEL
print('Step' + str(epoch + 1) )
print('Sum:' + str(np.sum(generated_image)))
print('Loss:'+ str(total_losses))
# save the picture
path = '%s/%d_outputs/%d_seed/%s_relative_weight' % (style_name,
iteration,
seed,
str(content_loss_weight))
if not os.path.exists(path):
os.makedirs(path)
filename = '%s/%d_outputs/%d_seed/%s_relative_weight/epoch_%d.png' % (style_name,
iteration,
seed,
str(content_loss_weight),
epoch+1)
generated_image = generated_image[0]
generated_image = np.clip(generated_image, 0, 255).astype('uint8')
imageio.imwrite(filename, generated_image)
if show == True:
plt.imshow(generated_image)
plt.axis('off')
# define the train function with seed 2019 and iteration 100 set by default
content_path = 'drive/My Drive/5242_project/content5.jpg'
Image.open(content_path)
# input the content image
style_path = 'drive/My Drive/5242_project/style2.jpg'
style_name = str(style_path.split("/")[-1].split(".")[0])
Image.open(style_path)
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
# build Style & Content Representations
content_loss_weight = 0.001
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
# define the relevant weight and compute the loss
Iteration = 100
train(iteration=100)
#training the model with 100 iterations
path = 'style2/100_outputs/2019_seed/0.001_relative_weight/epoch_100.png'
Image.open(path)
Iteration = 300
train(iteration=300)
#training the model with 300 iterations
path = 'style2/300_outputs/2019_seed/0.001_relative_weight/epoch_300.png'
Image.open(path)
Iteration = 500
train(iteration=500)
#training the model with 500 iterations
path = 'style2/500_outputs/2019_seed/0.001_relative_weight/epoch_500.png'
Image.open(path)
Iteration = 1000
train(iteration=1000)
#training the model with 1000 iterations
path = 'style2/1000_outputs/2019_seed/0.001_relative_weight/epoch_1000.png'
Image.open(path)
seed = 0
train(seed=0)
#training the model with seed 0
path = 'style2/100_outputs/0_seed/0.001_relative_weight/epoch_100.png'
Image.open(path)
seed = 2019
train(seed=2019)
#training the model with seed 2019
path = 'style2/100_outputs/2019_seed/0.001_relative_weight/epoch_100.png'
Image.open(path)
seed = 5242
train(seed = 5242)
#training the model with seed 5242
path = 'style2/100_outputs/5242_seed/0.001_relative_weight/epoch_100.png'
Image.open(path)
relative_weight = 0.01
tf.reset_default_graph()
# content_path = 'drive/My Drive/42 Project/content.jpg'
# style_path = 'drive/My Drive/42 Project/style.jpg'
# content_path = 'drive/My Drive/content.jpg'
# style_path = 'drive/My Drive/style.jpg'
content_path = 'drive/My Drive/5242_project/content5.jpg'
style_path = 'drive/My Drive/5242_project/style2.jpg'
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
content_loss_weight = 0.01
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
train()
#training the model with 0.01, 0.001 relative weight
path = 'style2/100_outputs/2019_seed/0.01_relative_weight/epoch_100.png'
Image.open(path)
relative_weight = 0.001
path = 'style2/100_outputs/2019_seed/0.001_relative_weight/epoch_100.png'
Image.open(path)
relative_weight = 0.0001
tf.reset_default_graph()
content_path = 'drive/My Drive/5242_project/content5.jpg'
style_path = 'drive/My Drive/5242_project/style2.jpg'
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
content_loss_weight = 0.0001
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
train()
#training the model with 0.0001 relative weight
path = 'style2/100_outputs/2019_seed/0.0001_relative_weight/epoch_100.png'
Image.open(path)
After parameter tuning process, we decide to use our model with 1000 iterations, 0.0001 relative weights and 2019 (default) seed.
tf.reset_default_graph()
content_path = 'drive/My Drive/5242_project/content5.jpg'
style_path = 'drive/My Drive/5242_project/style2.jpg'
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
content_loss_weight = 0.0001
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
train(iteration = 1000)
path = 'style2/1000_outputs/2019_seed/0.0001_relative_weight/epoch_1000.png'
Image.open(path)
tf.reset_default_graph()
content_path = 'drive/My Drive/5242_project/content2.jpg'
style_path = 'drive/My Drive/5242_project/style2.jpg'
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
content_loss_weight = 0.0001
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
train(iteration = 1000)
path = 'style2/1000_outputs/2019_seed/0.0001_relative_weight/epoch_1000.png'
Image.open(path)
tf.reset_default_graph()
content_path = 'drive/My Drive/5242_project/content10.jpg'
style_path = 'drive/My Drive/5242_project/style2.jpg'
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
content_loss_weight = 0.0001
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
train(iteration = 1000)
path = 'style2/1000_outputs/2019_seed/0.0001_relative_weight/epoch_1000.png'
Image.open(path)
tf.reset_default_graph()
content_path = 'drive/My Drive/5242_project/content5.jpg'
style_path = 'drive/My Drive/5242_project/style5.jpg'
style_name = str(style_path.split("/")[-1].split(".")[0])
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
content_loss_weight = 0.0001
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
train(iteration=1000)
path = 'style5/1000_outputs/2019_seed/0.0001_relative_weight/epoch_1000.png'
Image.open(path)
tf.reset_default_graph()
content_path = 'drive/My Drive/5242_project/content2.jpg'
style_path = 'drive/My Drive/5242_project/style5.jpg'
style_name = str(style_path.split("/")[-1].split(".")[0])
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
content_loss_weight = 0.0001
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
train(iteration=1000)
path = 'style5/1000_outputs/2019_seed/0.0001_relative_weight/epoch_1000.png'
Image.open(path)
tf.reset_default_graph()
content_path = 'drive/My Drive/5242_project/content10.jpg'
style_path = 'drive/My Drive/5242_project/style5.jpg'
style_name = str(style_path.split("/")[-1].split(".")[0])
image_width = 400
image_height = 300
MEAN_PIXEL = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))
style_image, content_image, noise_images = preprocess(content_path,style_path)
input_img = inp(image_width, image_height)
vgg = vgg_model(input_img)
content_loss_weight = 0.0001
style_loss_weight = 1
style_layers_weights = [0.5, 1.0, 1.5, 3.0, 4.0]
content_loss, style_loss, total_loss = losses()
train(iteration=1000)
path = 'style5/1000_outputs/2019_seed/0.0001_relative_weight/epoch_1000.png'
Image.open(path)